/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved.
 * Description: UDK mempool source file
 * Author: -
 * Create: 2021.5.11
 */
#include <unistd.h>
#include <stdint.h>
#include <inttypes.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>

#include "securec.h"
#include "udk.h"
#include "udk_common.h"
#include "udk_log.h"
#include "udk_ops.h"
#include "udk_tailq.h"
#include "udk_malloc.h"
#include "udk_mcfg.h"
#include "udk_mempool.h"

TAILQ_HEAD(udk_mempool_list, udk_tailq_entry);
static struct udk_tailq_elem udk_mempool_tailq = {
    .name = "UDK_MEMPOOL",
};
UDK_REGISTER_TAILQ(udk_mempool_tailq);

#define CACHE_FLUSHTHRESH_MULTIPLIER 1.5
#define UDK_CALC_CACHE_FLUSHTHRESH(c) ((typeof(c))((c)*CACHE_FLUSHTHRESH_MULTIPLIER))
#define UDK_ERRBUF_SZ 256

uint16_t g_udk_max_lcore = UDK_MAX_LCORE_DEFAULT;

/* get the greatest common divisor between m and n : gcd(m,n) = gcd(n,m mod n) */
static uint32_t udk_get_gcd(uint32_t m, uint32_t n)
{
    uint32_t r;

    if (m == 0) {
        return n;
    }

    if (n == 0) {
        return m;
    }

    if (m < n) {
        r = m;
        m = n;
        n = r;
    }

    while (n != 0) {
        r = m % n;
        m = n;
        n = r;
    }

    return m;
}

static void mempool_add_elem(struct udk_mempool *mp, udk_unused void *opaque, void *obj, udk_iova_t iova)
{
    struct udk_mempool_objhdr *hdr;

    /* set mempool ptr in header */
    hdr = UDK_PTR_SUB(obj, sizeof(*hdr));
    hdr->mp = mp;
    hdr->iova = iova;
    STAILQ_INSERT_TAIL(&mp->elt_list, hdr, next);
    mp->populated_size++;

#ifdef UDK_LIB_MEMPOOL_DEBUG
    hdr->cookie = UDK_MEMPOOL_HEADER_COOKIE2;
    tlr = __mempool_get_trailer(obj);
    tlr->cookie = UDK_MEMPOOL_TRAILER_COOKIE;
#endif

    return;
}

/* call obj_cb() for each mempool element */
uint32_t udk_mempool_obj_iter(struct udk_mempool *mp, udk_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
{
    struct udk_mempool_objhdr *hdr = NULL;
    void *obj = NULL;
    uint32_t n = 0;

    if (obj_cb == NULL) {
        return 0;
    }

    STAILQ_FOREACH(hdr, &mp->elt_list, next)
    {
        obj = (char *)hdr + sizeof(*hdr);
        obj_cb(mp, obj_cb_arg, obj, n);
        n++;
    }

    return n;
}

static uint32_t udk_optimize_object_size(uint32_t obj_size)
{
    uint32_t nrank, nchan;
    uint32_t new_obj_size;

    /* get number of channels */
    nchan = udk_memory_get_nchannel();
    if (nchan == 0) {
        nchan = UDK_DEFAULT_CHANNEL_NUM;
    }

    nrank = udk_memory_get_nrank();
    if (nrank == 0) {
        nrank = 1;
    }

    /* process new object size */
    new_obj_size = (obj_size + UDK_MEMPOOL_ALIGN_MASK) / UDK_MEMPOOL_ALIGN;
    while (udk_get_gcd(new_obj_size, nrank * nchan) != 1) {
        new_obj_size++;
    }
    return new_obj_size * UDK_MEMPOOL_ALIGN;
}

/* get the header, trailer and total size of a mempool element. */
static uint32_t udk_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct udk_mempool_objsz *sz)
{
    struct udk_mempool_objsz lsz;

    sz = (sz != NULL) ? sz : &lsz;

    sz->header_size = sizeof(struct udk_mempool_objhdr);
    if ((flags & UDK_MEMPOOL_F_NO_CACHE_ALIGN) == 0) {
        sz->header_size = UDK_ALIGN_CEIL(sz->header_size, UDK_MEMPOOL_ALIGN);
    }

#ifdef UDK_MEMPOOL_DEBUG
    sz->trailer_size = sizeof(struct udk_mempool_objtlr);
#else
    sz->trailer_size = 0;
#endif

    /* element size is 8 bytes-aligned at least */
    sz->elt_size = UDK_ALIGN_CEIL(elt_size, sizeof(uint64_t));

    /* expand trailer to next cache line */
    if ((flags & UDK_MEMPOOL_F_NO_CACHE_ALIGN) == 0) {
        sz->total_size = sz->header_size + sz->elt_size + sz->trailer_size;
        sz->trailer_size += ((UDK_MEMPOOL_ALIGN - (sz->total_size & UDK_MEMPOOL_ALIGN_MASK)) & UDK_MEMPOOL_ALIGN_MASK);
    }

    /* increase trailer to add padding between objects in order to spread them across memory channels/ranks */
    if ((flags & UDK_MEMPOOL_F_NO_SPREAD) == 0) {
        unsigned int new_size;
        new_size = udk_optimize_object_size(sz->header_size + sz->elt_size + sz->trailer_size);
        sz->trailer_size = (new_size - sz->header_size) - sz->elt_size;
    }

    /* this is the size of an object, including header and trailer */
    sz->total_size = sz->header_size + sz->elt_size + sz->trailer_size;

    return sz->total_size;
}

static void mempool_cache_init(struct udk_mempool_cache *cache, uint32_t size)
{
    cache->size = size;
    cache->flushthresh = UDK_CALC_CACHE_FLUSHTHRESH(size);
    cache->len = 0;
}

/* create an empty mempool */
struct udk_mempool *udk_mempool_create_empty(const char *name, uint32_t n, uint32_t elt_size, uint32_t cache_size,
    uint32_t private_data_size, int socket_id, uint32_t flags)
{
    char mz_name[UDK_MEMZONE_NAMESIZE];
    struct udk_mempool_list *mempool_list = NULL;
    struct udk_mempool *mp = NULL;
    struct udk_tailq_entry *te = NULL;
    const struct udk_memzone *mz = NULL;
    size_t mempool_size;
    uint32_t mz_flags = UDK_MEMZONE_1GB | UDK_MEMZONE_SIZE_HINT_ONLY;
    struct udk_mempool_objsz mp_objsz;
    unsigned lcore_id;
    int ret;

    /* compilation-time checks */
    UDK_BUILD_BUG_ON((sizeof(struct udk_mempool) & UDK_CACHE_LINE_MASK) != 0);
    UDK_BUILD_BUG_ON((sizeof(struct udk_mempool_cache) & UDK_CACHE_LINE_MASK) != 0);

    mempool_list = UDK_TAILQ_CAST(udk_mempool_tailq.head, udk_mempool_list);

    /* asked for zero items */
    if (n == 0) {
        return NULL;
    }

    /* asked cache too big */
    if (cache_size > UDK_MEMPOOL_CACHE_MAX_SIZE || UDK_CALC_CACHE_FLUSHTHRESH(cache_size) > n) {
        return NULL;
    }

    /* "no cache align" imply "no spread" */
    if (flags & UDK_MEMPOOL_F_NO_CACHE_ALIGN) {
        flags |= UDK_MEMPOOL_F_NO_SPREAD;
    }

    /* calculate mempool object sizes. */
    if (!udk_mempool_calc_obj_size(elt_size, flags, &mp_objsz)) {
        return NULL;
    }

    udk_mcfg_mempool_write_lock();

    /* reserve a memory zone for this mempool: private data is cache-aligned */
    private_data_size = (private_data_size + UDK_MEMPOOL_ALIGN_MASK) & (~UDK_MEMPOOL_ALIGN_MASK);

    te = udk_zmalloc(sizeof(*te), 0);
    if (te == NULL) {
        UDK_LOG(ERR, MEMPOOL, "Cannot allocate tailq entry!\n");
        goto exit_unlock;
    }

    mempool_size = (size_t)UDK_MEMPOOL_HEADER_SIZE(mp, cache_size);
    mempool_size += private_data_size;
    mempool_size = UDK_ALIGN_CEIL(mempool_size, UDK_MEMPOOL_ALIGN);

    ret = snprintf_s(mz_name, sizeof(mz_name), sizeof(mz_name) - 1, UDK_MEMPOOL_MZ_FORMAT, name);
    if (ret <= 0) {
        goto exit_unlock;
    }

    mz = udk_memzone_reserve(mz_name, mempool_size, socket_id, mz_flags);
    if (mz == NULL) {
        goto exit_unlock;
    }

    /* init the mempool structure */
    mp = mz->addr;
    (void)memset_s(mp, UDK_MEMPOOL_HEADER_SIZE(mp, cache_size), 0, UDK_MEMPOOL_HEADER_SIZE(mp, cache_size));
    ret = strcpy_s(mp->name, sizeof(mp->name), name);
    if (ret != 0) {
        goto exit_unlock;
    }

    mp->mz = mz;
    mp->flags = flags;
    mp->socket_id = socket_id;
    mp->size = n;
    mp->cache_size = cache_size; /* Size of default caches, zero means disabled. */
    mp->elt_size = mp_objsz.elt_size;
    mp->header_size = mp_objsz.header_size;
    mp->trailer_size = mp_objsz.trailer_size;
    mp->private_data_size = private_data_size;
    STAILQ_INIT(&mp->elt_list);
    STAILQ_INIT(&mp->mem_list);

    /* local_cache pointer is set even if cache_size is zero. The local_cache points to just past the elt_pa[] array. */
    mp->local_cache = (struct udk_mempool_cache *)UDK_PTR_ADD(mp, UDK_MEMPOOL_HEADER_SIZE(mp, 0));

    /* Init all default caches. */
    if (cache_size != 0) {
        for (lcore_id = 0; lcore_id < UDK_MAX_LCORE; lcore_id++) {
            mempool_cache_init(&mp->local_cache[lcore_id], cache_size);
        }
    }

    te->data = mp;

    udk_mcfg_tailq_write_lock();
    TAILQ_INSERT_TAIL(mempool_list, te, next);
    udk_mcfg_tailq_write_unlock();
    udk_mcfg_mempool_write_unlock();

    return mp;

exit_unlock:
    udk_mcfg_mempool_write_unlock();
    udk_free(te);
    udk_mempool_free(mp);
    return NULL;
}


void udk_mempool_put(struct udk_mempool *mp, void *obj)
{
    if (g_udk_reg_ops.mempool_put != NULL) {
        g_udk_reg_ops.mempool_put(mp, obj);
        return;
    }

    udk_mempool_put_bulk(mp, &obj, 1);
    return;
}

int udk_mempool_get_bulk(struct udk_mempool *mp, void **obj_table, uint32_t n)
{
    struct udk_mempool_cache *cache = NULL;

    if (g_udk_reg_ops.mempool_get_bulk != NULL) {
        return g_udk_reg_ops.mempool_get_bulk(mp, obj_table, n);
    }

    if (mp->cache_size > 0) {
        cache = udk_mempool_default_cache(mp, udk_lcore_id());
    }
    return udk_mempool_generic_get(mp, obj_table, n, cache);
}

uint32_t udk_mempool_avail_count(const struct udk_mempool *mp)
{
    uint32_t count;
    uint32_t lcore_id;

    if (g_udk_reg_ops.mempool_avail_count != NULL) {
        return g_udk_reg_ops.mempool_avail_count(mp);
    }

    count = udk_mempool_ops_get_count(mp);

    if (mp->cache_size == 0) {
        return count;
    }

    for (lcore_id = 0; lcore_id < UDK_MAX_LCORE; lcore_id++) {
        count += mp->local_cache[lcore_id].len;
    }

    /* As access to len is not locked, the count can be greater than size, so fix it */
    if (count > mp->size) {
        return mp->size;
    }
    return count;
}

uint32_t udk_mempool_in_use_count(const struct udk_mempool *mp)
{
    if (g_udk_reg_ops.mempool_in_use_count != NULL) {
        return g_udk_reg_ops.mempool_in_use_count(mp);
    }

    return mp->size - udk_mempool_avail_count(mp);
}

struct udk_mempool *udk_mempool_lookup(const char *name)
{
    struct udk_mempool *mp = NULL;
    struct udk_tailq_entry *te = NULL;
    struct udk_mempool_list *mempool_list = NULL;

    if (g_udk_reg_ops.mempool_lookup != NULL) {
        return g_udk_reg_ops.mempool_lookup(name);
    }

    mempool_list = UDK_TAILQ_CAST(udk_mempool_tailq.head, udk_mempool_list);

    udk_mcfg_mempool_read_lock();

    TAILQ_FOREACH(te, mempool_list, next)
    {
        mp = (struct udk_mempool *)te->data;
        if (strncmp(name, mp->name, UDK_MEMPOOL_NAMESIZE) == 0)
            break;
    }

    udk_mcfg_mempool_read_unlock();

    if (te == NULL) {
        return NULL;
    }

    return mp;
}

struct udk_mempool *udk_mempool_create(const char *name, uint32_t n, uint32_t elt_size, uint32_t cache_size,
    uint32_t private_data_size, udk_mempool_ctor_t *mp_init, void *mp_init_arg, udk_mempool_obj_cb_t *obj_init,
    void *obj_init_arg, int socket_id, uint32_t flags)
{
    int ret;
    struct udk_mempool *mp = NULL;

    if (g_udk_reg_ops.mempool_create != NULL) {
        return g_udk_reg_ops.mempool_create(name, n, elt_size, cache_size, private_data_size, mp_init, mp_init_arg,
            obj_init, obj_init_arg, socket_id, flags);
    }

    mp = udk_mempool_create_empty(name, n, elt_size, cache_size, private_data_size, socket_id, flags);
    if (mp == NULL) {
        return NULL;
    }

    /* examine the flags of 4 combinations of SP/SC/MP/MC, and set the correct index into the table of ops */
    if ((flags & UDK_MEMPOOL_F_SP_PUT) && (flags & UDK_MEMPOOL_F_SC_GET)) {
        ret = udk_mempool_set_ops_byname(mp, "ring_sp_sc", NULL);
    } else if (flags & UDK_MEMPOOL_F_SP_PUT) {
        ret = udk_mempool_set_ops_byname(mp, "ring_sp_mc", NULL);
    } else if (flags & UDK_MEMPOOL_F_SC_GET) {
        ret = udk_mempool_set_ops_byname(mp, "ring_mp_sc", NULL);
    } else {
        ret = udk_mempool_set_ops_byname(mp, "ring_mp_mc", NULL);
    }

    if (ret != 0) {
        goto fail;
    }

    /* call the mempool private initializer */
    if (mp_init != NULL) {
        mp_init(mp, mp_init_arg);
    }

    if (udk_mempool_populate_default(mp) < 0) {
        goto fail;
    }

    /* call the object initializers */
    if (obj_init != NULL) {
        (void)udk_mempool_obj_iter(mp, obj_init, obj_init_arg);
    }

    return mp;

 fail:
    udk_mempool_free(mp);
    return NULL;
}

/* Free memory chunks used by a mempool. Objects must be in pool */
static void udk_mempool_free_memchunks(struct udk_mempool *mp)
{
    struct udk_mempool_memhdr *mhdr = NULL;
    void *elt = NULL;

    while (!STAILQ_EMPTY(&mp->elt_list)) {
        (void)udk_mempool_ops_dequeue_bulk(mp, &elt, 1);
        (void)elt;
        STAILQ_REMOVE_HEAD(&mp->elt_list, next);
        mp->populated_size--;
    }

    while (!STAILQ_EMPTY(&mp->mem_list)) {
        mhdr = STAILQ_FIRST(&mp->mem_list);
        STAILQ_REMOVE_HEAD(&mp->mem_list, next);
        if (mhdr->free_cb != NULL)
            mhdr->free_cb(mhdr, mhdr->opaque);
        udk_free(mhdr);
        mp->nb_mem_chunks--;
    }
}

static int udk_mempool_ops_alloc_once(struct udk_mempool *mp)
{
    int ret;

    /* create the internal ring if not already done */
    if ((mp->flags & UDK_MEMPOOL_F_POOL_CREATED) == 0) {
        ret = udk_mempool_ops_alloc(mp);
        if (ret != 0) {
            return ret;
        }
        mp->flags |= UDK_MEMPOOL_F_POOL_CREATED;
    }
    return 0;
}

/* Add objects in the pool, using a physically contiguous memory zone.
 * Return the number of objects added, or a negative value on error.
 */
static int udk_mempool_populate_iova(struct udk_mempool *mp, char *vaddr, udk_iova_t iova, size_t len,
    udk_mempool_memchunk_free_cb_t *free_cb, void *opaque)
{
    int i = 0;
    size_t off;
    struct udk_mempool_memhdr *mhdr = NULL;
    int ret;

    ret = udk_mempool_ops_alloc_once(mp);
    if (ret != 0) {
        return ret;
    }

    /* mempool has already been populated */
    if (mp->populated_size >= mp->size) {
        return -ENOSPC;
    }

    mhdr = udk_zmalloc(sizeof(*mhdr), 0);
    if (mhdr == NULL) {
        return -ENOMEM;
    }

    mhdr->mp = mp;
    mhdr->addr = vaddr;
    mhdr->iova = iova;
    mhdr->len = len;
    mhdr->free_cb = free_cb;
    mhdr->opaque = opaque;

    if (mp->flags & UDK_MEMPOOL_F_NO_CACHE_ALIGN) {
        off = UDK_PTR_ALIGN_CEIL(vaddr, 0x8) - vaddr;
    } else {
        off = UDK_PTR_ALIGN_CEIL(vaddr, UDK_MEMPOOL_ALIGN) - vaddr;
    }

    if (off > len) {
        ret = 0;
        goto fail;
    }

    i = udk_mempool_ops_populate(mp, mp->size - mp->populated_size, (char *)vaddr + off,
        (iova == UDK_BAD_IOVA) ? UDK_BAD_IOVA : (iova + off), len - off, mempool_add_elem, NULL);
    if (i == 0) {
        /* not enough room to store one object */
        ret = 0;
        goto fail;
    }

    STAILQ_INSERT_TAIL(&mp->mem_list, mhdr, next);
    mp->nb_mem_chunks++;
    return i;

fail:
    udk_free(mhdr);
    return ret;
}

/* free a memchunk allocated with udk_memzone_reserve() */
static void udk_mempool_memchunk_mz_free(udk_unused struct udk_mempool_memhdr *mhdr, void *opaque)
{
    const struct udk_memzone *mz = opaque;
    (void)udk_memzone_free(mz);
    return;
}

#define PFN_MASK_SIZE 8
static udk_iova_t get_iova(const void *virtaddr)
{
    int fd = 0;
    int retval;
    uint64_t page, physaddr;
    unsigned long virt_pfn, page_size;
    off_t offset;
    char errbuf[UDK_ERRBUF_SZ] = {0};

    /* standard page size */
    page_size = getpagesize();
    if (page_size == 0) {
        UDK_LOG(ERR, MEMPOOL, "page size is 0\n");
        return UDK_BAD_IOVA;
    }

    fd = open("/proc/self/pagemap", O_RDONLY);
    if (fd < 0) {
        (void)strerror_r(errno, errbuf, sizeof(errbuf));
        UDK_LOG(ERR, MEMPOOL, "Cannot open /proc/self/pagemap: %s\n", errbuf);
        return UDK_BAD_IOVA;
    }

    virt_pfn = ((uintptr_t)virtaddr) / page_size;
    offset = (off_t)(sizeof(uint64_t) * virt_pfn);
    if (lseek(fd, offset, SEEK_SET) == (off_t)-1) {
        (void)strerror_r(errno, errbuf, sizeof(errbuf));
        UDK_LOG(ERR, MEMPOOL, "Seek error in /proc/self/pagemap: %s\n", errbuf);
        close(fd);
        return UDK_BAD_IOVA;
    }

    retval = (int)read(fd, &page, PFN_MASK_SIZE);
    close(fd);
    if (retval < 0) {
        (void)strerror_r(errno, errbuf, sizeof(errbuf));
        UDK_LOG(ERR, MEMPOOL, "Cannot read /proc/self/pagemap: %s\n", errbuf);
        return UDK_BAD_IOVA;
    } else if (retval != PFN_MASK_SIZE) {
        UDK_LOG(ERR, MEMPOOL, "Read %d bytes from /proc/self/pagemap but expected %d:\n", retval, PFN_MASK_SIZE);
        return UDK_BAD_IOVA;
    }

    /* the pfn (page frame number) are bits 0-54 (see pagemap.txt in linux Documentation) */
    if ((page & 0x7fffffffffffffULL) == 0) {
        return UDK_BAD_IOVA;
    }

    physaddr = ((page & 0x7fffffffffffffULL) * page_size) + ((uintptr_t)virtaddr % page_size);
    return physaddr;
}

/* Populate the mempool with a virtual area. Return the number of objects added, or a negative value on error. */
static int udk_mempool_populate_virt(struct udk_mempool *mp, const struct udk_memzone *mz,
    udk_mempool_memchunk_free_cb_t *free_cb, size_t pg_sz)
{
    int ret;
    udk_iova_t iova;
    size_t off, phys_len;
    int cnt = 0;
    char *addr = mz->addr;
    size_t len = mz->len;

    if (mp->flags & UDK_MEMPOOL_F_NO_IOVA_CONTIG) {
        return udk_mempool_populate_iova(mp, addr, UDK_BAD_IOVA, len, free_cb, (void *)mz);
    }

    for (off = 0; off < len && mp->populated_size < mp->size; off += phys_len) {
        iova = get_iova(addr + off);

        /* populate with the largest group of contiguous pages */
        for (phys_len = UDK_MIN((size_t)(UDK_PTR_ALIGN_CEIL(addr + off + 1, pg_sz) - (addr + off)), len - off);
            off + phys_len < len; phys_len = UDK_MIN(phys_len + pg_sz, len - off)) {
            udk_iova_t iova_tmp;

            iova_tmp = get_iova(addr + off + phys_len);
            if (iova_tmp == UDK_BAD_IOVA || iova_tmp != iova + phys_len) {
                break;
            }
        }

        ret = udk_mempool_populate_iova(mp, addr + off, iova, phys_len, free_cb, (void *)mz);
        if (ret == 0) {
            continue;
        }

        if (ret < 0) {
            goto out;
        }
        /* not to call the callback in next iter */
        free_cb = NULL;
        cnt += ret;
    }

    return cnt;

 out:
    udk_mempool_free_memchunks(mp);
    return ret;
}

/* Get the minimal page size used in a mempool before populating it. */
int udk_mempool_get_page_size(udk_unused struct udk_mempool *mp, size_t *pg_sz)
{
    int need_iova_contig_obj;
    int alloc_in_ext_mem;
    int ret;

    /* check if we can retrieve a valid socket ID */
    ret = udk_malloc_heap_socket_is_external(mp->socket_id);
    if (ret < 0) {
        return -EINVAL;
    }

    alloc_in_ext_mem = (ret == 1);
    need_iova_contig_obj = !(mp->flags & UDK_MEMPOOL_F_NO_IOVA_CONTIG);
    if (need_iova_contig_obj == 0) {
        *pg_sz = 0;
    } else if ((udk_has_hugepages() != 0) || (alloc_in_ext_mem != 0)) {
        *pg_sz = udk_page_size();
    } else {
        *pg_sz = getpagesize();
    }

    return 0;
}

static int mempool_populate_default_chunk(struct udk_mempool *mp, uint32_t mz_id, size_t min_chunk,
    size_t mem_size, uint32_t align, size_t pg_sz)
{
    uint32_t mz_flags = UDK_MEMZONE_1GB | UDK_MEMZONE_SIZE_HINT_ONLY;
    const struct udk_memzone *mz = NULL;
    char mz_name[UDK_MEMZONE_NAMESIZE];
    int need_iova_contig_obj;
    udk_iova_t iova;
    int ret;

    ret = snprintf_s(mz_name, sizeof(mz_name), sizeof(mz_name) - 1, UDK_MEMPOOL_MZ_FORMAT "_%u", mp->name, mz_id);
    if ((ret < 0) || (ret >= (int)sizeof(mz_name))) {
        return -ENAMETOOLONG;
    }

    /* if we're trying to reserve contiguous memory, add appropriate memzone flag. */
    if ((min_chunk == mem_size) || (mp->flags & UDK_MEMPOOL_F_IOVA_CONTIG)) {
        mz_flags |= UDK_MEMZONE_IOVA_CONTIG;
    }

    mz = udk_memzone_reserve_aligned(mz_name, mem_size, mp->socket_id, mz_flags, align);
    /* don't try reserving with 0 size if we were asked to reserve IOVA-contiguous memory. */
    if ((min_chunk < mem_size) && (mz == NULL)) {
        /* not enough memory, retry with the biggest zone we have */
        UDK_LOG(ERR, MEMPOOL, "Alloc failed, min_chunk:%zu, mz_size:%zu, align:%u, mz_name:%s.\n",
            min_chunk, mem_size, align, mz_name);
        mz = udk_memzone_reserve_aligned(mz_name, 0, mp->socket_id, mz_flags, align);
    }
    if (mz == NULL) {
        return -ENOMEM;
    }

    if (mz->len < min_chunk) {
        (void)udk_memzone_free(mz);
        return -ENOMEM;
    }

    need_iova_contig_obj = !(mp->flags & UDK_MEMPOOL_F_NO_IOVA_CONTIG);
    if (need_iova_contig_obj != 0) {
        iova = mz->iova;
    } else {
        iova = UDK_BAD_IOVA;
    }

    if ((need_iova_contig_obj == 0) || (mz_flags & UDK_MEMZONE_IOVA_CONTIG)) {
        ret = udk_mempool_populate_iova(mp, mz->addr, iova, mz->len, udk_mempool_memchunk_mz_free, (void *)mz);
    } else {
        ret = udk_mempool_populate_virt(mp, mz, udk_mempool_memchunk_mz_free, pg_sz);
    }
    if (ret == 0) {
        ret = -ENOBUFS;
    }

    if (ret < 0) {
        (void)udk_memzone_free(mz);
    }

    return ret;
}

/* Default function to populate the mempool: allocate memory in memzones, and populate them.
 * Return the number of objects added, or a negative value on error.
 */
int udk_mempool_populate_default(struct udk_mempool *mp)
{
    uint32_t mz_id, n, pg_shift = 0;
    ssize_t mem_size;
    size_t align, pg_sz, min_chunk_size;
    int ret;

    if ((mp->socket_id != UDK_SOCKET_ID_ANY) || ((mp->flags & UDK_MEMPOOL_F_NO_IOVA_CONTIG) != 0)) {
        UDK_LOG(ERR, MEMPOOL, "Invalid argument to populate, socket_id: %d, flags: %u!\n",
                mp->socket_id, mp->flags);
        return -EINVAL;
    }

    ret = udk_mempool_ops_alloc_once(mp);
    if (ret != 0) {
        return ret;
    }

    /* mempool must not be populated */
    if (mp->nb_mem_chunks != 0) {
        UDK_LOG(ERR, MEMPOOL, "mempool has been populated\n");
        return -EEXIST;
    }

    ret = udk_mempool_get_page_size(mp, &pg_sz);
    if (ret < 0) {
        return ret;
    }

    if (pg_sz != 0) {
        pg_shift = udk_bsf32((uint32_t)pg_sz);
    }

    for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= (uint32_t)ret) {
        mem_size = udk_mempool_ops_calc_mem_size(mp, n, pg_shift, &min_chunk_size, &align);
        if (mem_size < 0) {
            ret = (int)mem_size;
            goto fail;
        }

        ret = mempool_populate_default_chunk(mp, mz_id, min_chunk_size, mem_size, (uint32_t)align, pg_sz);
        if (ret < 0) {
            goto fail;
        }
    }

    return (int)mp->size;

fail:
    udk_mempool_free_memchunks(mp);
    return ret;
}

void udk_mempool_free(struct udk_mempool *mp)
{
    if (g_udk_reg_ops.mempool_free != NULL) {
        g_udk_reg_ops.mempool_free(mp);
        return;
    }

    struct udk_mempool_list *mp_list = NULL;
    struct udk_tailq_entry *te_iter = NULL;

    if (mp == NULL) {
        return;
    }

    mp_list = UDK_TAILQ_CAST(udk_mempool_tailq.head, udk_mempool_list);
    udk_mcfg_tailq_write_lock();
    /* find tailq entry */
    TAILQ_FOREACH(te_iter, mp_list, next)
    {
        if (te_iter->data == (void *)mp)
            break;
    }

    if (te_iter != NULL) {
        TAILQ_REMOVE(mp_list, te_iter, next);
        udk_free(te_iter);
    }
    udk_mcfg_tailq_write_unlock();

    udk_mempool_free_memchunks(mp);
    udk_mempool_ops_free(mp);
    (void)udk_memzone_free(mp->mz);
    return;
}

#ifdef UDK_MEMPOOL_DEBUG
static void udk_mempool_check_cookies(const struct udk_mempool *mp, void * const * obj_table_const, uint32_t n,
    int free)
{
    struct udk_mempool_objhdr *hdr = NULL;
    struct udk_mempool_objtlr *tlr = NULL;
    uint64_t cookie;
    void *obj = NULL;
    void **obj_table = NULL;

    obj_table = (void **)obj_table_const;

    while (n--) {
        obj = obj_table[n];

        if (udk_mempool_from_obj(obj) != mp) {
            udk_panic("MEMPOOL: object is owned by another mempool\n");
        }

        hdr = udk_mempool_get_header(obj);
        cookie = hdr->cookie;

        if (free == 0x0) {
            if (cookie != UDK_MEMPOOL_HEADER_COOKIE1) {
                UDK_LOG(CRIT, MEMPOOL, "obj[0x%lx], mempool[0x%lx], cookie[%" PRIx64 "]\n", udk_get_ptr(obj),
                    udk_get_ptr(mp), cookie);
                udk_panic("MEMPOOL: bad header cookie (put)\n");
            }
            hdr->cookie = UDK_MEMPOOL_HEADER_COOKIE2;
        } else if (free == 0x1) {
            if (cookie != UDK_MEMPOOL_HEADER_COOKIE2) {
                UDK_LOG(CRIT, MEMPOOL, "obj[0x%lx], mempool[0x%lx], cookie[%" PRIx64 "]\n", udk_get_ptr(obj),
                    udk_get_ptr(mp), cookie);
                udk_panic("MEMPOOL: bad header cookie (get)\n");
            }
            hdr->cookie = UDK_MEMPOOL_HEADER_COOKIE1;
        } else if (free == 0x2) {
            if (cookie != UDK_MEMPOOL_HEADER_COOKIE1 && UDK_MEMPOOL_HEADER_COOKIE2) {
                UDK_LOG(CRIT, MEMPOOL, "obj[0x%lx], mempool[0x%lx], cookie[%" PRIx64 "]\n", udk_get_ptr(obj),
                    udk_get_ptr(mp), cookie);
                udk_panic("MEMPOOL: bad header cookie (audit)\n");
            }
        }

        tlr = udk_mempool_get_trailer(obj);
        cookie = tlr->cookie;
        if (cookie != UDK_MEMPOOL_TRAILER_COOKIE) {
            UDK_LOG(CRIT, MEMPOOL, "obj[0x%lx], mempool[0x%lx], cookie[%" PRIx64 "]\n", udk_get_ptr(obj),
                udk_get_ptr(mp), cookie);
            udk_panic("MEMPOOL: bad trailer cookie\n");
        }
    }
}

static void udk_mempool_obj_audit(struct udk_mempool *mp, udk_unused void *opaque, void *obj, udk_unused uint32_t idx)
{
    udk_mempool_check_cookies(mp, &obj, 1, 0x2);
}

static void udk_mempool_audit_cookies(struct udk_mempool *mp)
{
    uint32_t num = udk_mempool_obj_iter(mp, udk_mempool_obj_audit, NULL);
    if (num != mp->size) {
        udk_panic("udk_mempool_obj_iter(mempool=0x%lx, size=%u) iterated only over %u elements\n", udk_get_ptr(mp),
            mp->size, num);
    }
}
#else
#define udk_mempool_audit_cookies(mp) \
    do {                              \
    } while (0)
#endif

static uint32_t udk_mempool_dump_cache(FILE *f, const struct udk_mempool *mp)
{
    uint32_t lcore_id, cache_cnt;
    uint32_t count = 0;

    (void)fprintf(f, "  internal cache infos:\n");
    (void)fprintf(f, "    cache_size=%u\n", mp->cache_size);

    if (mp->cache_size == 0) {
        return count;
    }

    for (lcore_id = 0; lcore_id < UDK_MAX_LCORE; ++lcore_id) {
        cache_cnt = mp->local_cache[lcore_id].len;
        (void)fprintf(f, "    cache_count[%u]=%u\n", lcore_id, cache_cnt);
        count += cache_cnt;
    }
    (void)fprintf(f, "    total_cache_count=%u\n", count);
    return count;
}

static void udk_mempool_audit_cache(const struct udk_mempool *mp)
{
    uint32_t lcore_id;
    const struct udk_mempool_cache *cache = NULL;

    if (mp->cache_size == 0) {
        return;
    }

    for (lcore_id = 0; lcore_id < UDK_MAX_LCORE; ++lcore_id) {
        cache = &mp->local_cache[lcore_id];
        if (cache->len > cache->flushthresh) {
            UDK_LOG(CRIT, MEMPOOL, "badness on cache[%u]\n", lcore_id);
            udk_panic("MEMPOOL: invalid cache len\n");
        }
    }
}

static void udk_mempool_audit(struct udk_mempool *mp)
{
    udk_mempool_audit_cache(mp);
    udk_mempool_audit_cookies(mp);
}

void udk_mempool_dump(FILE *f, struct udk_mempool *mp)
{
    struct udk_mempool_memhdr *memhdr = NULL;
    struct udk_mempool_ops *ops = NULL;
    uint32_t common_cnt, cache_cnt;
    size_t mem_len = 0;
#ifdef UDK_MEMPOOL_DEBUG
    struct udk_mempool_info info;
    struct udk_mempool_debug_stats sum;
    uint32_t lcore_id;
#endif

    UDK_ASSERT(f != NULL);
    UDK_ASSERT(mp != NULL);

    (void)fprintf(f, "mempool <%s>@0x%lx\n", mp->name, udk_get_ptr(mp));
    (void)fprintf(f, "  pool=0x%lx\n", udk_get_ptr(mp->pool_data));
    (void)fprintf(f, "  iova=0x%" PRIx64 "\n", mp->mz->iova);
    (void)fprintf(f, "  flags=%x\n", mp->flags);
    (void)fprintf(f, "  socket_id=%d\n", mp->socket_id);
    (void)fprintf(f, "  size=%u\n", mp->size);
    (void)fprintf(f, "  elt_size=%u\n", mp->elt_size);
    (void)fprintf(f, "  header_size=%u\n", mp->header_size);
    (void)fprintf(f, "  trailer_size=%u\n", mp->trailer_size);
    (void)fprintf(f, "  total_obj_size=%u\n", mp->header_size + mp->elt_size + mp->trailer_size);
    (void)fprintf(f, "  private_data_size=%u\n", mp->private_data_size);
    (void)fprintf(f, "  ops_index=%d\n", mp->ops_index);
    ops = udk_mempool_get_ops(mp->ops_index);
    (void)fprintf(f, "  ops_name: <%s>\n", (ops != NULL) ? ops->name : "NA");
    (void)fprintf(f, "  populated_size=%u\n", mp->populated_size);
    (void)fprintf(f, "  nb_mem_chunks=%u\n", mp->nb_mem_chunks);

    STAILQ_FOREACH(memhdr, &mp->mem_list, next)
    {
        mem_len += memhdr->len;
    }
    if (mem_len != 0) {
        (void)fprintf(f, "  avg bytes/object=%#Lf\n", (long double)mem_len / mp->size);
    }

    cache_cnt = udk_mempool_dump_cache(f, mp);
    common_cnt = udk_mempool_ops_get_count(mp);
    if (cache_cnt + common_cnt > mp->size) {
        common_cnt = mp->size - cache_cnt;
    }
    (void)fprintf(f, "  common_pool_count=%u\n", common_cnt);

#ifdef UDK_MEMPOOL_DEBUG
    /* debug: statistics */
    udk_mempool_ops_get_info(mp, &info);
    memset_s(&sum, sizeof(sum), 0, sizeof(sum));

    for (lcore_id = 0; lcore_id < UDK_MAX_LCORE; ++lcore_id) {
        sum.put_objs += mp->stats[lcore_id].put_objs;
        sum.put_bulk += mp->stats[lcore_id].put_bulk;
        sum.get_success_objs += mp->stats[lcore_id].get_success_objs;
        sum.get_success_bulk += mp->stats[lcore_id].get_success_bulk;
        sum.get_fail_objs += mp->stats[lcore_id].get_fail_objs;
        sum.get_fail_bulk += mp->stats[lcore_id].get_fail_bulk;
        sum.get_success_blks += mp->stats[lcore_id].get_success_blks;
        sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks;
    }
    (void)fprintf(f, "  stats:\n");
    (void)fprintf(f, "    put_bulk=%" PRIu64 "\n", sum.put_bulk);
    (void)fprintf(f, "    put_objs=%" PRIu64 "\n", sum.put_objs);
    (void)fprintf(f, "    get_success_bulk=%" PRIu64 "\n", sum.get_success_bulk);
    (void)fprintf(f, "    get_success_objs=%" PRIu64 "\n", sum.get_success_objs);
    (void)fprintf(f, "    get_fail_bulk=%" PRIu64 "\n", sum.get_fail_bulk);
    (void)fprintf(f, "    get_fail_objs=%" PRIu64 "\n", sum.get_fail_objs);
    if (info.contig_block_size > 0) {
        (void)fprintf(f, "    get_success_blks=%" PRIu64 "\n", sum.get_success_blks);
        (void)fprintf(f, "    get_fail_blks=%" PRIu64 "\n", sum.get_fail_blks);
    }
#endif
    udk_mempool_audit(mp);
}
